'''
Created on May 7, 2011

@author: Jialai_Zhu
'''
from mc import site
from db_zq import extract_fake_zq
#from rarfile import *
from zipfile import *
from ptn import *
def FILE_PATH_EX(path,sub=''):
    import os
    BASE=os.path.dirname(p=__file__)
    p=os.path.join(BASE,sub)
    p=os.path.join(p,path)    
    p=os.path.abspath(p)
    return p
DOWNLOAD_FOLDER=FILE_PATH_EX('download','../')
try:
    import os
    print DOWNLOAD_FOLDER
    os.mkdir(DOWNLOAD_FOLDER)
except:
    pass
def htmldecode(s):
    ret=reg_ptn(u'(&#x([\d|\w]{4});)+').search_all(s)
    r=[]
    for i in ret:
        u=u''
        for j in reg_ptn(u'&#x([\d|\w]{4});',1).search_all(i.extract_part,):
            print j.extract_part
            u=u+unichr(int('0x'+j.extract_part,16))  
        r.append(u) 
    return r

extractor=extract_fake_zq()
def html_charset(string):
    for encode in ['gb2312','gbk','utf8']:
        try:
            unicode(string,encode)
            char_set=encode
            return char_set
        except:
            pass
    return None
class extract_result():
    def __init__(self):
        self.phone=[]
        self.cert=''
        self.company=[]
        self.js_redir=[]
        self.is_result_done=False
    @staticmethod
    def func_append_list(list,w,equal):
        for l in list:
            if equal(l,w)==True:
                return
        list.append(w)     
    def add_phone(self,phonelist):
        for w in phonelist:        
            extract_result.func_append_list(self.phone, w, lambda x,y:x==y)
        pass
    def add_cert(self,cert):
        if cert==None:return
        self.cert=cert
    def add_sec_company(self,company):
        if company==None or company['type']==u'':return True
        extract_result.func_append_list(self.company, company, lambda x,y:x['type']==y['type'])
#        if company['type']==u''or company['type']==u'domain' or company['type']==u'server':
#            if self.is_result_done==False:
#                
#        else:
#            extract_result.func_append_list(self.company, company, lambda x,y:x['type']==y['type'])
#            self.is_result_done=True
    def add_redir_js(self,url):
        if len(url)==0:return
        extract_result.func_append_list(self.js_redir, url, lambda x,y:x==y)
    def result_done(self):
        return self.is_result_done
    def result_type(self):
        type=u''
        b302=''
        bserver=''
        bdomain=''
        ret={'qs_f':'','qs_s':''}
        for company in self.company:            
            if len(company['type'])==0:continue
            if company['type']==u'domain':bdomain=True
            elif company['type']==u'server':bserver=True
            elif company['type']==u'302'   :b302=True
            else:
                try:
                    if len(ret[company['name']]):
                        pass
                except:
                    ret[company['type']]=company['name']
        ret['site_302']=b302        
        ret['site_500']=bserver
        ret['site_domain']=bdomain
        return ret
g_extract_result=None
class zq_site(site): 
    def urltags(self):
        return {
                "iframe": "src",
                "frame": "src",
                "a":"src"
                }
    def __init__(self,folder=None,request=None,domain=None,url=None,level=0,maxlevel=3,type='',item=None):
        if item!=None:
            self.item=item
            self.level=0
        else:
            site.__init__(self, folder, request, domain, url, level, maxlevel, type)
    def parser(self):
        import os
        import tempfile

        dir=tempfile.mkdtemp(suffix='200',dir='temp')
        zip_file=os.path.join(DOWNLOAD_FOLDER,self.item['zip'])
        ZipFile(zip_file).extractall(path=dir)
        def t(path):
            for f in os.listdir(path):
                fbfile=os.path.join(path,f)
                if os.path.isdir(fbfile):
                    if t(fbfile)==True:
                        break
                else:
                    string=self.unicode_html(open(fbfile,'rb').read())
                    if self.parserhtml(string)==True:
                        return True
        try:
            print dir
            t(dir)
            pass
        except:
            pass        
    def site_class(self): 
        return zq_site
    def allfilter(self,link): 
        if link.tag=='iframe':return True
        if link.tag=='frame':return True
        if link.tag=='a':return True
        return False
    def result(self):
        global g_extract_result
        try:
            if g_extract_result==None:
                g_extract_result=extract_result()
        except:
            g_extract_result=extract_result()
            pass
        return g_extract_result
    def redir_js(self,stringU):
        try:
#            if self.soup==None:return
            if self.level!=0:return
            import re
    #        tags=self.soup.findAll(name='script',text='location')
            r=re.search(u'location\s*=\s*"\s*([a-zA-z]+://(\w+(-\w+)*)(\.(\w+(-\w+)*))*(\?\S*)?)',stringU)
            if r!=None:
                print r.group(0)
                print r.group(1)   
                self.result().add_sec_company({'name':r.group(1),'type':'302'}) 
                self.result().add_redir_js(url=r.group(1))
        except Exception,ex:
            print ex
            pass    
    def htmlfilter(self):
        string=self.html()
        self.soup=None   
        self.parserhtml(string)     
    def unicode_html(self,string):        
        if isinstance(string,unicode):
            return string
        c=html_charset(string)
        try:
            if c!=None: 
                stringU=unicode(string,c)
            else:
                stringU=unicode(string)
        except:
            stringU=u''
        return stringU     
    def parserhtml(self,string):
        try:		
            stringU=self.unicode_html(string)	
            html_decode=htmldecode(stringU)
            def parse_phone_cert(stringU):
                self.redir_js(stringU)
                phonelist=extractor.phone_num2(stringU)
                if len(phonelist):
                    self.result().add_phone(phonelist)
                    pass
                sec_cert=extractor.sec_cert(stringU)
                if sec_cert!=None:
                    self.result().add_cert(sec_cert.match_string)

            parse_phone_cert(stringU)	
            for decode_string in html_decode:
                parse_phone_cert(decode_string)

            
            for company in extractor.gettype(stringU,True):
                self.result().add_sec_company(company)
#            if company['type']==u''or company['type']==u'domain' or company['type']==u'server' or company['type']==u'302':
#                self.result().add_sec_company(company)
#            else:
#                self.result().add_sec_company({'name':company['name'],'type':'qs_f'})
#                self.result().add_sec_company({'name':company['type'],'type':'qs_s'})
            return False
        except Exception,ex:
            return False
    
    @staticmethod
    def download(url,folder='./'):
        try:
            import string 
            s=zq_site(url=url,maxlevel=1,folder=folder)
            s.all()
            s.save()
            s.zip(DOWNLOAD_FOLDER)
            if s.is_sucess():
                code='200'
            else:
                code=str(s.BrowserError.code)
            return {'result':s.result(),'zip':s.zipfile,'code':code,'url':url}
        except Exception,ex:
            print ex
        return None
    

from db_zq import DB_QS_S
from db_zq import DB_QS_F
from db_zq import DB_QUERY_PHONE
from db_zq import DB_QUERY_CERT
DB_QUERY_DAILY=None
def update_db_with_result(item):
    site_status=g_extract_result.result_type()
    DB_QUERY_DAILY.update_item(item, site_status)
    if len(site_status['qs_s']) or len(site_status['qs_f']):
        qs=DB_QS_S.find({'name':site_status['qs_s']}, 1)
        if qs==None:
            DB_QS_S.insert({'name':site_status['qs_s']})
            
        qs=DB_QS_F.find({'name':site_status['qs_f']}, 1)
        if qs==None:
            DB_QS_F.insert({'name':site_status['qs_f']}) 
                           
        for p in g_extract_result.phone:
            phone=DB_QUERY_PHONE.find({'num':p}, 1)
            zip_array=[]
            domain_array=[]
            
            try: zip_array=phone['zip']
            except:pass
            extract_result.func_append_list(zip_array, item['zip'], lambda x,y:x==y)
            
            
            try: domain_array=phone['domain']
            except:pass
            extract_result.func_append_list(domain_array, item['domain'], lambda x,y:x==y)
            
            
            if phone!=None:
                DB_QUERY_PHONE.update_item(phone, {'zip':zip_array,'domain':domain_array})
            else:
                DB_QUERY_PHONE.insert({'zip':zip_array,'num':p,'domain':domain_array})
                
        if len(g_extract_result.cert):
            p=g_extract_result.cert
            cert=DB_QUERY_CERT.find({'cert':p}, 1)
            domain=[]
            try:domain=cert['domain']
            except:pass
            extract_result.func_append_list(domain, item['domain'], lambda x,y:x==y)                
            if cert!=None:                    
                DB_QUERY_CERT.update_item(cert, {'domain':domain})
            else:
                DB_QUERY_CERT.insert({'domain':domain,'cert':p})                
def cralwer(item):
    try:
        try:
            if len(item['code']) and (int(item['code'])!=20000):
                return
        except:
            pass
        url=item['url']
#        print url
        
        
        def download(url,dir):
            global g_extract_result
            g_extract_result=None
            ret=zq_site.download(url=url,folder=dir)
            if ret==None:
                DB_QUERY_DAILY.update_item(item,{'code':'oops'})
                return -1
            data={}
            print ret['code']
            item['code']=data['code']=ret['code']
            code=int(data['code'])
            item['zip']=data['zip']=ret['zip']
            DB_QUERY_DAILY.update_item(item,data)
            update_db_with_result(item)
            return code
        if len(url)==0:
            url='http://'+item['domain']
            print 'domain',url  
        import string
        dir=string.replace(url, ':', '+')
        dir=string.replace(dir, '/', '_')
        dir=string.replace(dir, '?', '_')
        import os
        dir=os.path.join(DOWNLOAD_FOLDER,dir)
        if len(dir)>100:
            dir=dir[0:100]        
        ret=download(url,dir)
        if len(item['url'])==0:
            return
        if ret==200:
            return
        if ret>=1000:
            return 
        if (ret>=400 and ret<600) or ret==250:
            print "retry domain"
            download('http://'+item['domain'],dir)
            pass            
            
    except Exception,ex:
        pass
import multiprocessing
import sys
if sys.platform == 'win32':
    import multiprocessing.reduction    # make sockets pickable/inheritable
def   child_process(item):
    print item['url']
    try:
        cralwer(item)
    except Exception,ex:
        print ex
    
def   child_process2(item):
#    print item['url']
    code=int(item['code'])
    if code<10000:
        try:
#            print item['url']
            global g_extract_result
            g_extract_result=None
            zq_site(item).parser()
            update_db_with_result(item)                    
        except Exception,ex:
            print ex
            pass    
def main(processes=4,func=None):
    items=[]
    for object in DB_QUERY_DAILY.find({}, 0):
        items.append(object)
    if processes==1:
        for item in items:
            cralwer(item)
        return     
    pool = multiprocessing.Pool(processes=processes)
    for item in items:
#        print item['url']
        pool.apply_async(func, (item,))
    pool.close()
    pool.join()   
    
def process_zip(bCrawler=True):
    items=[]
    for object in DB_QUERY_DAILY.find({}, 0):
        items.append(object)
    f=gb2312file('log.txt','wb')
    for item in items:
        try:
            if bCrawler:
                cralwer(item)
            else:
                child_process2(item)
        except:
            pass
#        return     

def test(): 
    string=open('index.txt','rb').read()    
    def unicode_html(string):
        import  BeautifulSoup
        BeautifulSoup.BeautifulSoup
        if isinstance(string,unicode):
            return string
        import BeautifulSoup
        stringU=None
        char_set=None
        for encode in ['gb2312','gbk','utf8']:
            try:
                stringU=unicode(string,encode)
                char_set=encode
                break
            except:
                pass
        if stringU==None:
            try:
                stringU=unicode(string)   
            except:
                pass    
        return stringU
    extract_fake_zq(ptn='zq_type2.txt').gettype(unicode_html(string))
import os
import os
import getopt, sys
class Config(object):
    def __init__(self):
        self.process=1
        self.db='test'
class CommandLine(object):
    def __init__(self,config):       
        self.opts, self.args = getopt.getopt(sys.argv[1:], self.short_option(), self.long_option())
        self.config=config
        self.ProcessCommand()
    def ProcessCommand(self):
        for o, a in self.opts:
            self.ProcessCommandOther(o, a)
    def short_option(self):return "s:d:m:"
    def long_option(self):return ["p=","db="]
    def ProcessCommandOther(self,o, a):
        if o=="--db":
            self.config.db=a
        if o=='--p':
            self.config.process=int(a)    
def run(process_num,db_name):
    import db_result
    db_result.db_init(db_name)
    global DB_QUERY_DAILY
    DB_QUERY_DAILY=db_result.DB_QUERY_DAILY
    main(process_num,child_process)    
if __name__ == '__main__':
    import sys
    setting =Config()
    c       =CommandLine(setting)
    run(setting.process,setting.db)